##################################################
# the module contains some util function for 
# operating url
##################################################
# version 0.1 2010 Mar 13

import urllib
import urllib2

# the file name can not cantian:\,/,:,,*,?,",<,>,|
# the url can not contains:',;,1=1,|,<,>
# the chars that url can, but file name can not is: \,/,:,*,?,"
# the chars that file name can, but url can not is: '
def url2filename(url):
    url = url.replace("\\", "'")
    url = url.replace("/", "'")
    url = url.replace(";", "'")
    url = url.replace("*", "'")
    url = url.replace("?", "'")
    url = url.replace('"', "'")
    return url

def quote(url):
    url = del_duplicate_header('http://', url)
    header = 'http://'
    p = url.find(header)
    if p != -1:
        u = url[p + len(header):len(url)]
    else:
        u = url
    u = urllib.quote(u)
    return 'http://%s' % u

def get_page_content(url):
    user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
    headers = { 'User-Agent' : user_agent }
    req = urllib2.Request(url, '', headers)
    response = urllib2.urlopen(req)
    return response.read()

def del_duplicate_header(header, url):
    if url[0: len(header)] == header:
        url = url[len(header): len(url)]
    return url

if __name__ == '__main__':
    url = """http://\\www.baidu.com;*?\""""
    print url2filename(url)

    url = 'http://www.dianping.com/search/keyword/2/10_\xe6\x9c\x8b\xe5\x8f\x8b\xe8\x81\x9a\xe9\xa4\x90'
    print quote(url)

    url = 'http://http://www.baidu.com'
    url = del_duplicate_header('http://', url)
    print url
